# -*- coding: UTF-8 -*-
# Author: Damon(wuud1)
# CreateDate:
# Message: 尝试自己写断点续爬功能

import pymysql
import redis
import time

db_config = dict(
    host="127.0.0.1",
    port=3306,
    user="root",
    passwd="123456",
    db="my_data",
    charset="utf8"
)
conn = pymysql.connect(**db_config)
curs = conn.cursor(pymysql.cursors.SSCursor)
sql = '''select pid from 0716_pid'''
curs.execute(sql)

# redis 配置
r = redis.Redis(host='127.0.0.1', port=6379, db=0)

for pid in curs:
    if r.sismember('pid_set', pid):
        print "已经抓取, 跳过"
        continue
    print "抓取过程...."
    time.sleep(1)
    r.sadd('pid_set', pid)
    r.save()





